{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# HER2 One Scanner - Aperio FDA\n",
    "\n",
    "- 3-Fold (50/50) split, No Holdout Set\n",
    "- Truth = Categorical from Mean of 7 continuous scores \n",
    "- Epoch at automatic Stop when loss<.001 change \n",
    "- LeNet model, 10 layers, Dropout (0.7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "from keras.callbacks import EarlyStopping\n",
    "from PIL import Image\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D, Lambda\n",
    "from keras.layers import Dense\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from keras.utils import np_utils\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.metrics import roc_curve, auc, classification_report\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from collections import Counter\n",
    "import csv\n",
    "import cv2\n",
    "import scipy\n",
    "import os\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "#For single scanner\n",
    "BASE_PATH = '/home/diam/Desktop/HER2_data_categorical/Aperio_FDA/'\n",
    "#BASE PATH for working from home:\n",
    "#BASE_PATH = '/home/OSEL/Desktop/HER2_data_categorical/'\n",
    "#epochs = 10\n",
    "batch_size = 32\n",
    "num_classes = 3\n",
    "#epochs = 35"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get Data - Practice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This is the version from Ravi's code:\n",
    "\n",
    "#FDA\n",
    "#X_FDA = []\n",
    "#idx_FDA = []\n",
    "#for index, image_filename in list(enumerate(BASE_PATH)):\n",
    "#\timg_file = cv2.imread(BASE_PATH + '/' + image_filename)\n",
    "#\tif img_file is not None:\n",
    "\t\t#img_file = smisc.imresize(arr = img_file, size = (600,760,3))\n",
    "#\t\timg_file = smisc.imresize(arr = img_file, size = (120,160,3))\t\t\n",
    "#\t\timg_arr = np.asarray(img_file)\n",
    "#\t\tX_FDA.append(img_arr)\n",
    "#\t\tidx_FDA.append(index)\n",
    "\n",
    "#X_FDA = np.asarray(X_FDA)\n",
    "#idx_FDA = np.asarray(idx_FDA)\n",
    "#random.seed(rs)\n",
    "#random_id = random.sample(idx_FDA, len(idx_FDA)/2)\n",
    "#random_FDA = []\n",
    "#for i in random_id:\n",
    "#\trandom_FDA.append(X_FDA[i])\n",
    "\n",
    "#random_FDA = np.asarray(random_FDA)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get Data - Real"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data(folder):\n",
    "    X = []\n",
    "    y = []\n",
    "    filenames = []\n",
    "\n",
    "    for hclass in os.listdir(folder):\n",
    "        if not hclass.startswith('.'):\n",
    "            if hclass in [\"1\"]:\n",
    "                label = 1\n",
    "            else: #label must be 1 or 2\n",
    "                if hclass in [\"2\"]:\n",
    "                    label = 2\n",
    "                else:\n",
    "                    label = 3\n",
    "            for image_filename in os.listdir(folder + hclass):\n",
    "                filename = folder + hclass + '/' + image_filename\n",
    "                img_file = cv2.imread(folder + hclass + '/' + image_filename)\n",
    "                \n",
    "                if img_file is not None:\n",
    "                    img_file = scipy.misc.imresize(arr=img_file, size=(120, 160, 3))\n",
    "                    img_arr = np.asarray(img_file)\n",
    "                    X.append(img_arr)\n",
    "                    y.append(label)\n",
    "                    filenames.append(filename)\n",
    "\n",
    "    X = np.asarray(X)\n",
    "    y = np.asarray(y)\n",
    "    z = np.asarray(filenames)\n",
    "    return X,y,z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:20: DeprecationWarning: `imresize` is deprecated!\n",
      "`imresize` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.\n",
      "Use ``skimage.transform.resize`` instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "241\n",
      "241\n",
      "241\n"
     ]
    }
   ],
   "source": [
    "X, y, z = get_data(BASE_PATH)\n",
    "\n",
    "\n",
    "#print(X)\n",
    "#print(y)\n",
    "#print(z)\n",
    "print(len(X))\n",
    "print(len(y))\n",
    "print(len(z))\n",
    "\n",
    "#INTEGER ENCODE\n",
    "#https://machinelearningmastery.com/how-to-one-hot-encode-sequence-data-in-python/\n",
    "encoder = LabelEncoder()\n",
    "y_cat = np_utils.to_categorical(encoder.fit_transform(y))\n",
    "#print(y_cat)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Old Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#encoder = LabelEncoder()\n",
    "#encoder.fit(y)\n",
    "\n",
    "#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=10)\n",
    "\n",
    "#encoded_y_train = encoder.transform(y_train)\n",
    "#encoded_y_test = encoder.transform(y_test)\n",
    "\n",
    "#y_train = np_utils.to_categorical(encoded_y_train)\n",
    "#y_test = np_utils.to_categorical(encoded_y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fit Model with K-Fold X-Val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n",
      "KFold(n_splits=2, random_state=5, shuffle=True)\n"
     ]
    }
   ],
   "source": [
    "#Need to remove random_state before running multiple thousands of times and averaging\n",
    "kf = KFold(n_splits = 2, random_state=5, shuffle=True)\n",
    "print(kf.get_n_splits(y))\n",
    "print(kf)\n",
    "\n",
    "\n",
    "for train_index_full, test_index in kf.split(y):\n",
    "    val_index = np.random.choice(train_index_full, 21, replace=False)\n",
    "    train_index = [x for x in train_index_full if x not in val_index]\n",
    "    X_train, X_val, X_test = X[train_index], X[val_index], X[test_index]\n",
    "    y_train, y_val, y_test = y[train_index], y[val_index], y[test_index]\n",
    "    z_train, z_val, z_test = z[train_index], z[val_index], z[test_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_index)\n",
    "#len(test_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold #1\n",
      "(12.0, <type 'numpy.float64'>)\n",
      "36\n",
      "('augX_train:', 9)\n",
      "9\n",
      "('auginX_train:', 1)\n",
      "1\n",
      "<keras.preprocessing.image.NumpyArrayIterator object at 0x7f0950303290>\n",
      "Train on 108 samples, validate on 12 samples\n",
      "Epoch 1/1000\n",
      "108/108 [==============================] - 2s 20ms/step - loss: 1.1538 - acc: 0.3796 - val_loss: 0.9855 - val_acc: 0.6667\n",
      "Epoch 2/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 1.0152 - acc: 0.4630 - val_loss: 0.8559 - val_acc: 0.6667\n",
      "Epoch 3/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.9770 - acc: 0.5370 - val_loss: 0.9621 - val_acc: 0.6667\n",
      "Epoch 4/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 1.0507 - acc: 0.5741 - val_loss: 1.4296 - val_acc: 0.2500\n",
      "Epoch 5/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 1.2101 - acc: 0.4352 - val_loss: 0.7701 - val_acc: 0.6667\n",
      "Epoch 6/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.8940 - acc: 0.6204 - val_loss: 0.9034 - val_acc: 0.7500\n",
      "Epoch 7/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.7917 - acc: 0.7870 - val_loss: 0.7090 - val_acc: 0.7500\n",
      "Epoch 8/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 1.0547 - acc: 0.5463 - val_loss: 0.7024 - val_acc: 0.8333\n",
      "Epoch 9/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.7030 - acc: 0.8056 - val_loss: 0.7859 - val_acc: 0.7500\n",
      "Epoch 10/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.7752 - acc: 0.6667 - val_loss: 0.6495 - val_acc: 0.7500\n",
      "Epoch 11/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 1.1130 - acc: 0.6574 - val_loss: 0.7913 - val_acc: 0.8333\n",
      "Epoch 12/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.7258 - acc: 0.7315 - val_loss: 0.6494 - val_acc: 0.7500\n",
      "Epoch 13/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.6312 - acc: 0.7778 - val_loss: 1.1967 - val_acc: 0.3333\n",
      "Epoch 14/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.7285 - acc: 0.6944 - val_loss: 0.6220 - val_acc: 0.7500\n",
      "Epoch 15/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.5599 - acc: 0.8611 - val_loss: 0.5228 - val_acc: 0.8333\n",
      "Epoch 16/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.7997 - acc: 0.6944 - val_loss: 0.5617 - val_acc: 0.8333\n",
      "Epoch 17/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.5077 - acc: 0.8796 - val_loss: 0.5755 - val_acc: 0.8333\n",
      "Epoch 18/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.6400 - acc: 0.7685 - val_loss: 0.5523 - val_acc: 0.7500\n",
      "Epoch 19/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.5015 - acc: 0.8611 - val_loss: 0.7038 - val_acc: 0.7500\n",
      "Epoch 20/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 1.0722 - acc: 0.6759 - val_loss: 0.5537 - val_acc: 0.8333\n",
      "Epoch 21/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.5443 - acc: 0.8241 - val_loss: 0.4790 - val_acc: 0.7500\n",
      "Epoch 22/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.5307 - acc: 0.8611 - val_loss: 1.4065 - val_acc: 0.4167\n",
      "Epoch 23/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.6110 - acc: 0.7593 - val_loss: 0.9183 - val_acc: 0.5000\n",
      "Epoch 24/1000\n",
      "108/108 [==============================] - 0s 1ms/step - loss: 0.5014 - acc: 0.8611 - val_loss: 0.5748 - val_acc: 0.7500\n",
      "Epoch 25/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.5273 - acc: 0.8148 - val_loss: 0.8136 - val_acc: 0.7500\n",
      "Epoch 26/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.5910 - acc: 0.8148 - val_loss: 1.1060 - val_acc: 0.5000\n",
      "Epoch 27/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.4727 - acc: 0.8611 - val_loss: 0.5801 - val_acc: 0.7500\n",
      "Epoch 28/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.9552 - acc: 0.7130 - val_loss: 0.5844 - val_acc: 0.8333\n",
      "Epoch 29/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.5335 - acc: 0.8333 - val_loss: 0.5051 - val_acc: 0.8333\n",
      "Epoch 30/1000\n",
      "108/108 [==============================] - 0s 1ms/step - loss: 0.5573 - acc: 0.8611 - val_loss: 1.0339 - val_acc: 0.5000\n",
      "Epoch 31/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.4151 - acc: 0.8426 - val_loss: 0.6989 - val_acc: 0.7500\n",
      "Epoch 32/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.4928 - acc: 0.8333 - val_loss: 0.6001 - val_acc: 0.7500\n",
      "Epoch 33/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.5182 - acc: 0.8426 - val_loss: 0.4880 - val_acc: 0.8333\n",
      "Epoch 34/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.4098 - acc: 0.8704 - val_loss: 0.7800 - val_acc: 0.6667\n",
      "Epoch 35/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.4176 - acc: 0.8704 - val_loss: 0.4470 - val_acc: 0.7500\n",
      "Epoch 36/1000\n",
      "108/108 [==============================] - 0s 1ms/step - loss: 0.9062 - acc: 0.6852 - val_loss: 0.7425 - val_acc: 0.5833\n",
      "Epoch 37/1000\n",
      "108/108 [==============================] - 0s 1ms/step - loss: 0.6010 - acc: 0.8889 - val_loss: 0.6087 - val_acc: 0.7500\n",
      "Epoch 38/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.4044 - acc: 0.8704 - val_loss: 0.7941 - val_acc: 0.7500\n",
      "Epoch 39/1000\n",
      "108/108 [==============================] - 0s 1ms/step - loss: 0.3708 - acc: 0.8796 - val_loss: 0.4567 - val_acc: 0.7500\n",
      "Epoch 40/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.5572 - acc: 0.8148 - val_loss: 0.5622 - val_acc: 0.7500\n",
      "Epoch 41/1000\n",
      "108/108 [==============================] - 0s 1ms/step - loss: 0.4299 - acc: 0.9167 - val_loss: 1.2188 - val_acc: 0.5000\n",
      "Epoch 42/1000\n",
      "108/108 [==============================] - 0s 1ms/step - loss: 0.3759 - acc: 0.8889 - val_loss: 1.1447 - val_acc: 0.6667\n",
      "Epoch 43/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.3224 - acc: 0.8981 - val_loss: 0.7312 - val_acc: 0.7500\n",
      "Epoch 44/1000\n",
      "108/108 [==============================] - 0s 1ms/step - loss: 0.3974 - acc: 0.8426 - val_loss: 0.6196 - val_acc: 0.7500\n",
      "Epoch 45/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.2940 - acc: 0.9074 - val_loss: 2.5187 - val_acc: 0.5000\n",
      "Epoch 46/1000\n",
      "108/108 [==============================] - 0s 1ms/step - loss: 0.3587 - acc: 0.8426 - val_loss: 0.8823 - val_acc: 0.7500\n",
      "Epoch 47/1000\n",
      "108/108 [==============================] - 0s 1ms/step - loss: 0.3515 - acc: 0.8704 - val_loss: 1.1285 - val_acc: 0.5000\n",
      "Epoch 48/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.6106 - acc: 0.7778 - val_loss: 1.1821 - val_acc: 0.5000\n",
      "Epoch 49/1000\n",
      "108/108 [==============================] - 0s 1ms/step - loss: 0.3559 - acc: 0.8796 - val_loss: 0.5417 - val_acc: 0.8333\n",
      "Epoch 50/1000\n",
      "108/108 [==============================] - 0s 1ms/step - loss: 0.4266 - acc: 0.8611 - val_loss: 0.7965 - val_acc: 0.6667\n",
      "Epoch 51/1000\n",
      "108/108 [==============================] - 0s 1ms/step - loss: 0.2799 - acc: 0.9074 - val_loss: 0.5072 - val_acc: 0.8333\n",
      "Epoch 52/1000\n",
      "108/108 [==============================] - 0s 1ms/step - loss: 0.6229 - acc: 0.8148 - val_loss: 0.6838 - val_acc: 0.7500\n",
      "Epoch 53/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.4450 - acc: 0.8519 - val_loss: 0.8008 - val_acc: 0.7500\n",
      "Epoch 54/1000\n",
      "108/108 [==============================] - 0s 2ms/step - loss: 0.3730 - acc: 0.8889 - val_loss: 1.0698 - val_acc: 0.6667\n",
      "Epoch 55/1000\n",
      "108/108 [==============================] - 0s 1ms/step - loss: 0.3144 - acc: 0.8889 - val_loss: 0.8464 - val_acc: 0.7500\n",
      "Epoch 56/1000\n",
      "108/108 [==============================] - 0s 1ms/step - loss: 0.3231 - acc: 0.9074 - val_loss: 1.5929 - val_acc: 0.5833\n",
      "Epoch 57/1000\n",
      "108/108 [==============================] - 0s 1ms/step - loss: 0.8155 - acc: 0.7407 - val_loss: 0.5012 - val_acc: 0.8333\n",
      "Epoch 58/1000\n",
      "108/108 [==============================] - 0s 1ms/step - loss: 0.4106 - acc: 0.8704 - val_loss: 0.6408 - val_acc: 0.7500\n",
      "Epoch 59/1000\n",
      "108/108 [==============================] - 0s 1ms/step - loss: 0.4062 - acc: 0.8796 - val_loss: 1.0829 - val_acc: 0.5000\n",
      "Epoch 60/1000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "108/108 [==============================] - 0s 2ms/step - loss: 0.3319 - acc: 0.8981 - val_loss: 0.9952 - val_acc: 0.6667\n",
      "Epoch 00060: early stopping\n",
      "Fold Score (accuracy): 0.793388429752\n",
      "[[11  0  0]\n",
      " [56 11  0]\n",
      " [ 3 40  0]]\n",
      "[[ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 1.  0.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 1.  0.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 1.  0.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]]\n",
      "[[  2.06493642e-04   1.03029057e-01   8.96764457e-01]\n",
      " [  1.79017574e-01   8.04631829e-01   1.63505170e-02]\n",
      " [  3.88104830e-18   1.40055461e-04   9.99859929e-01]\n",
      " [  1.81812763e-01   7.97909081e-01   2.02781335e-02]\n",
      " [  3.52146141e-02   7.07307279e-01   2.57478118e-01]\n",
      " [  2.06935973e-27   2.25849117e-08   1.00000000e+00]\n",
      " [  9.19089317e-02   8.80021691e-01   2.80693285e-02]\n",
      " [  9.42147215e-20   6.17663954e-06   9.99993801e-01]\n",
      " [  1.72862574e-01   8.12535465e-01   1.46019040e-02]\n",
      " [  6.65803270e-08   3.63962241e-02   9.63603735e-01]\n",
      " [  6.41031191e-02   6.94321513e-01   2.41575301e-01]\n",
      " [  4.79421665e-15   4.11168076e-05   9.99958873e-01]\n",
      " [  3.12337621e-11   2.36228356e-04   9.99763787e-01]\n",
      " [  1.83959261e-01   7.97973812e-01   1.80669501e-02]\n",
      " [  1.83472723e-01   7.98316479e-01   1.82108060e-02]\n",
      " [  1.55279542e-06   4.57005091e-02   9.54297900e-01]\n",
      " [  1.80351064e-01   8.02051783e-01   1.75972227e-02]\n",
      " [  1.72887176e-01   8.02664638e-01   2.44481694e-02]\n",
      " [  9.87962470e-04   3.58231694e-01   6.40780330e-01]\n",
      " [  1.85335189e-01   7.97806919e-01   1.68579426e-02]\n",
      " [  2.58462591e-04   1.16988845e-01   8.82752657e-01]\n",
      " [  2.04418791e-29   5.72683012e-09   1.00000000e+00]\n",
      " [  2.05950402e-02   6.47318900e-01   3.32086146e-01]\n",
      " [  1.33263975e-01   8.43634307e-01   2.31017340e-02]\n",
      " [  1.09746542e-28   1.16843148e-08   1.00000000e+00]\n",
      " [  7.57156684e-08   1.13754552e-02   9.88624454e-01]\n",
      " [  5.41554056e-02   7.74450541e-01   1.71394035e-01]\n",
      " [  5.09762950e-02   8.97918344e-01   5.11053465e-02]\n",
      " [  2.90440039e-06   2.94640716e-02   9.70533013e-01]\n",
      " [  1.66741148e-01   8.15185666e-01   1.80731062e-02]\n",
      " [  1.65366665e-01   8.21083128e-01   1.35502480e-02]\n",
      " [  2.03647768e-13   2.47001968e-04   9.99752939e-01]\n",
      " [  9.80803147e-02   8.86529684e-01   1.53900180e-02]\n",
      " [  1.77397266e-01   8.06976795e-01   1.56259611e-02]\n",
      " [  1.74763441e-01   8.10541868e-01   1.46946777e-02]\n",
      " [  1.02754369e-01   8.81362557e-01   1.58830695e-02]\n",
      " [  1.51880607e-27   4.47543478e-08   1.00000000e+00]\n",
      " [  1.67846739e-01   8.13622057e-01   1.85311344e-02]\n",
      " [  8.20523798e-02   8.62026572e-01   5.59210964e-02]\n",
      " [  1.83717072e-01   7.99199343e-01   1.70836244e-02]\n",
      " [  3.76440754e-15   1.19718497e-04   9.99880314e-01]\n",
      " [  1.16293104e-05   1.69197053e-01   8.30791295e-01]\n",
      " [  1.12996575e-04   2.40159601e-01   7.59727359e-01]\n",
      " [  6.03914959e-03   6.24286532e-01   3.69674325e-01]\n",
      " [  1.66381165e-01   8.15212607e-01   1.84062421e-02]\n",
      " [  1.86787248e-01   7.95074701e-01   1.81381274e-02]\n",
      " [  6.39349609e-18   1.43139105e-05   9.99985695e-01]\n",
      " [  1.57772063e-24   2.05995647e-07   9.99999762e-01]\n",
      " [  7.66155135e-04   3.27642858e-01   6.71590984e-01]\n",
      " [  1.60707719e-20   3.98506457e-03   9.96014953e-01]\n",
      " [  1.72326639e-01   8.13027740e-01   1.46456473e-02]\n",
      " [  2.23367885e-02   8.42317820e-01   1.35345340e-01]\n",
      " [  1.71388909e-01   8.04644883e-01   2.39661839e-02]\n",
      " [  1.64458811e-01   8.22995365e-01   1.25458287e-02]\n",
      " [  2.55810209e-02   9.46232975e-01   2.81859897e-02]\n",
      " [  3.23114311e-03   7.49314904e-01   2.47453958e-01]\n",
      " [  8.30862224e-02   6.95817888e-01   2.21095845e-01]\n",
      " [  3.83025385e-03   4.20219511e-01   5.75950205e-01]\n",
      " [  4.05627788e-12   4.58187307e-04   9.99541879e-01]\n",
      " [  1.75372750e-01   8.04696143e-01   1.99311133e-02]\n",
      " [  2.31844187e-02   8.33849430e-01   1.42966121e-01]\n",
      " [  1.77118868e-01   8.07386637e-01   1.54945087e-02]\n",
      " [  1.64892361e-01   8.08693528e-01   2.64141113e-02]\n",
      " [  1.77433982e-01   8.01290095e-01   2.12759245e-02]\n",
      " [  1.82344303e-01   8.01680446e-01   1.59752499e-02]\n",
      " [  1.64014503e-01   8.16490829e-01   1.94946863e-02]\n",
      " [  1.70305534e-10   1.04821054e-03   9.98951793e-01]\n",
      " [  1.61550805e-01   8.19763839e-01   1.86854023e-02]\n",
      " [  6.86415717e-22   2.38374469e-06   9.99997616e-01]\n",
      " [  5.37623464e-29   1.06840776e-08   1.00000000e+00]\n",
      " [  1.85168847e-01   7.93953896e-01   2.08773017e-02]\n",
      " [  1.75102964e-01   8.09680939e-01   1.52161457e-02]\n",
      " [  1.78048119e-01   8.04071724e-01   1.78801157e-02]\n",
      " [  1.73330247e-01   8.04404199e-01   2.22655255e-02]\n",
      " [  1.11901597e-13   3.19418526e-04   9.99680638e-01]\n",
      " [  1.67431237e-04   3.01869869e-01   6.97962642e-01]\n",
      " [  1.45381838e-01   8.25252056e-01   2.93661002e-02]\n",
      " [  9.07528922e-02   8.58646691e-01   5.06003909e-02]\n",
      " [  1.51358917e-01   8.26159775e-01   2.24813800e-02]\n",
      " [  1.61657099e-25   4.34710756e-08   1.00000000e+00]\n",
      " [  1.44141629e-01   8.14176023e-01   4.16823104e-02]\n",
      " [  2.92887371e-02   7.06321895e-01   2.64389396e-01]\n",
      " [  7.86457979e-15   3.44357279e-04   9.99655604e-01]\n",
      " [  2.36587077e-02   9.09229815e-01   6.71114549e-02]\n",
      " [  1.81327373e-01   7.97987163e-01   2.06854437e-02]\n",
      " [  6.67408085e-06   1.79791421e-01   8.20201874e-01]\n",
      " [  1.75456449e-01   8.06774318e-01   1.77691635e-02]\n",
      " [  1.79857522e-01   8.03620040e-01   1.65224373e-02]\n",
      " [  6.89660965e-06   1.00844644e-01   8.99148405e-01]\n",
      " [  1.82903573e-01   8.00638318e-01   1.64580680e-02]\n",
      " [  6.25522160e-12   8.20376948e-02   9.17962372e-01]\n",
      " [  1.54130623e-01   8.02183092e-01   4.36863378e-02]\n",
      " [  5.26154181e-19   4.73393084e-06   9.99995232e-01]\n",
      " [  3.11833085e-03   4.23323452e-01   5.73558211e-01]\n",
      " [  2.62187067e-02   5.70394397e-01   4.03386861e-01]\n",
      " [  1.06641926e-01   8.34525049e-01   5.88330217e-02]\n",
      " [  1.18471786e-01   7.96931088e-01   8.45970809e-02]\n",
      " [  6.93017989e-03   6.51837885e-01   3.41231912e-01]\n",
      " [  1.45506382e-01   8.38308454e-01   1.61852445e-02]\n",
      " [  5.64283796e-19   3.19158653e-06   9.99996781e-01]\n",
      " [  4.19439094e-19   3.05745743e-05   9.99969482e-01]\n",
      " [  1.81704164e-01   7.90217042e-01   2.80788820e-02]\n",
      " [  1.11490414e-17   1.42590306e-05   9.99985695e-01]\n",
      " [  5.36728051e-13   8.46802373e-04   9.99153137e-01]\n",
      " [  1.07166275e-01   8.37415099e-01   5.54186702e-02]\n",
      " [  1.86137811e-07   8.71592686e-02   9.12840486e-01]\n",
      " [  2.45065466e-02   8.50984275e-01   1.24509223e-01]\n",
      " [  8.99521634e-04   1.65348858e-01   8.33751559e-01]\n",
      " [  8.84547234e-02   8.89530957e-01   2.20143385e-02]\n",
      " [  2.02534811e-09   6.58661639e-03   9.93413389e-01]\n",
      " [  1.84209317e-01   7.98558235e-01   1.72324702e-02]\n",
      " [  1.97233145e-26   1.09004397e-07   9.99999881e-01]\n",
      " [  1.31632730e-01   8.04640412e-01   6.37268573e-02]\n",
      " [  5.67523450e-08   1.97571263e-01   8.02428663e-01]\n",
      " [  1.69314407e-02   9.01731789e-01   8.13368261e-02]\n",
      " [  1.27080952e-07   1.50044747e-02   9.84995425e-01]\n",
      " [  1.50274247e-01   8.25306535e-01   2.44193021e-02]\n",
      " [  6.73860273e-11   1.29393290e-03   9.98706102e-01]\n",
      " [  1.81554424e-12   3.89358262e-04   9.99610722e-01]\n",
      " [  6.03836728e-03   3.40609312e-01   6.53352320e-01]\n",
      " [  1.35374539e-18   2.17465786e-05   9.99978304e-01]]\n",
      "[2 1 2 1 1 2 1 2 1 2 1 2 2 1 1 2 1 1 2 1 2 2 1 1 2 2 1 1 2 1 1 2 1 1 1 1 2\n",
      " 1 1 1 2 2 2 1 1 1 2 2 2 2 1 1 1 1 1 1 1 2 2 1 1 1 1 1 1 1 2 1 2 2 1 1 1 1\n",
      " 2 2 1 1 1 2 1 1 2 1 1 2 1 1 2 1 2 1 2 2 1 1 1 1 1 2 2 1 2 2 1 2 1 2 1 2 1\n",
      " 2 1 2 1 2 1 2 2 2 2]\n",
      "fold #2\n",
      "(12.0, <type 'numpy.float64'>)\n",
      "44\n",
      "('augX_train:', 11)\n",
      "11\n",
      "('auginX_train:', 1)\n",
      "1\n",
      "<keras.preprocessing.image.NumpyArrayIterator object at 0x7f08dd485c90>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 109 samples, validate on 12 samples\n",
      "Epoch 1/1000\n",
      "109/109 [==============================] - 2s 19ms/step - loss: 1.6855 - acc: 0.4312 - val_loss: 1.1012 - val_acc: 0.4167\n",
      "Epoch 2/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 1.0350 - acc: 0.5229 - val_loss: 1.0420 - val_acc: 0.4167\n",
      "Epoch 3/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 1.1595 - acc: 0.2844 - val_loss: 0.9155 - val_acc: 0.8333\n",
      "Epoch 4/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.9823 - acc: 0.6147 - val_loss: 0.7628 - val_acc: 0.4167\n",
      "Epoch 5/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.9934 - acc: 0.5505 - val_loss: 0.7857 - val_acc: 0.5833\n",
      "Epoch 6/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 1.2801 - acc: 0.4404 - val_loss: 0.8124 - val_acc: 0.5833\n",
      "Epoch 7/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.8317 - acc: 0.6972 - val_loss: 1.0514 - val_acc: 0.4167\n",
      "Epoch 8/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.7974 - acc: 0.7064 - val_loss: 0.5294 - val_acc: 0.7500\n",
      "Epoch 9/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.8871 - acc: 0.6055 - val_loss: 0.8717 - val_acc: 0.5000\n",
      "Epoch 10/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.8089 - acc: 0.6972 - val_loss: 0.3199 - val_acc: 0.9167\n",
      "Epoch 11/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.6236 - acc: 0.7706 - val_loss: 0.6716 - val_acc: 0.7500\n",
      "Epoch 12/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.8298 - acc: 0.6422 - val_loss: 0.4091 - val_acc: 0.9167\n",
      "Epoch 13/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.5842 - acc: 0.7798 - val_loss: 0.4847 - val_acc: 0.8333\n",
      "Epoch 14/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 1.1628 - acc: 0.6147 - val_loss: 1.0256 - val_acc: 0.4167\n",
      "Epoch 15/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.8537 - acc: 0.7064 - val_loss: 0.5026 - val_acc: 0.8333\n",
      "Epoch 16/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.6359 - acc: 0.7982 - val_loss: 0.5021 - val_acc: 0.8333\n",
      "Epoch 17/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.7012 - acc: 0.7431 - val_loss: 0.3264 - val_acc: 0.9167\n",
      "Epoch 18/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.6783 - acc: 0.7706 - val_loss: 0.4482 - val_acc: 0.8333\n",
      "Epoch 19/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.5425 - acc: 0.8257 - val_loss: 0.6287 - val_acc: 0.7500\n",
      "Epoch 20/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.6675 - acc: 0.7339 - val_loss: 0.3048 - val_acc: 0.9167\n",
      "Epoch 21/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.6548 - acc: 0.7431 - val_loss: 0.3620 - val_acc: 0.8333\n",
      "Epoch 22/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.7822 - acc: 0.7339 - val_loss: 0.3590 - val_acc: 0.9167\n",
      "Epoch 23/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.6265 - acc: 0.7982 - val_loss: 0.8397 - val_acc: 0.6667\n",
      "Epoch 24/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.6121 - acc: 0.7982 - val_loss: 0.2024 - val_acc: 1.0000\n",
      "Epoch 25/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.6484 - acc: 0.7615 - val_loss: 0.5959 - val_acc: 0.7500\n",
      "Epoch 26/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.5443 - acc: 0.8257 - val_loss: 0.2487 - val_acc: 0.9167\n",
      "Epoch 27/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.7397 - acc: 0.7156 - val_loss: 0.7834 - val_acc: 0.7500\n",
      "Epoch 28/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.5993 - acc: 0.8257 - val_loss: 0.1869 - val_acc: 1.0000\n",
      "Epoch 29/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.5396 - acc: 0.8165 - val_loss: 1.1218 - val_acc: 0.5833\n",
      "Epoch 30/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.6290 - acc: 0.7706 - val_loss: 0.3052 - val_acc: 0.9167\n",
      "Epoch 31/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.4469 - acc: 0.8624 - val_loss: 1.1985 - val_acc: 0.5833\n",
      "Epoch 32/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 1.0065 - acc: 0.6422 - val_loss: 0.3871 - val_acc: 0.8333\n",
      "Epoch 33/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.4806 - acc: 0.8532 - val_loss: 0.1769 - val_acc: 1.0000\n",
      "Epoch 34/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.5090 - acc: 0.8073 - val_loss: 0.2279 - val_acc: 0.9167\n",
      "Epoch 35/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.4649 - acc: 0.8257 - val_loss: 0.7556 - val_acc: 0.7500\n",
      "Epoch 36/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.5571 - acc: 0.8073 - val_loss: 0.2227 - val_acc: 0.9167\n",
      "Epoch 37/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.3845 - acc: 0.8807 - val_loss: 0.6209 - val_acc: 0.7500\n",
      "Epoch 38/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.7047 - acc: 0.7339 - val_loss: 0.3528 - val_acc: 0.9167\n",
      "Epoch 39/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.4496 - acc: 0.8440 - val_loss: 0.5009 - val_acc: 0.7500\n",
      "Epoch 40/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.4767 - acc: 0.8165 - val_loss: 0.2764 - val_acc: 0.9167\n",
      "Epoch 41/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.3700 - acc: 0.8624 - val_loss: 0.2715 - val_acc: 0.9167\n",
      "Epoch 42/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.4841 - acc: 0.8165 - val_loss: 0.3157 - val_acc: 0.9167\n",
      "Epoch 43/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.3850 - acc: 0.8624 - val_loss: 0.4570 - val_acc: 0.8333\n",
      "Epoch 44/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.3782 - acc: 0.8532 - val_loss: 0.7427 - val_acc: 0.7500\n",
      "Epoch 45/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.4924 - acc: 0.8257 - val_loss: 1.0177 - val_acc: 0.6667\n",
      "Epoch 46/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.4827 - acc: 0.8073 - val_loss: 0.8576 - val_acc: 0.7500\n",
      "Epoch 47/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.7967 - acc: 0.7798 - val_loss: 1.1958 - val_acc: 0.6667\n",
      "Epoch 48/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.5816 - acc: 0.8165 - val_loss: 0.1497 - val_acc: 1.0000\n",
      "Epoch 49/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.3483 - acc: 0.8716 - val_loss: 0.4056 - val_acc: 0.9167\n",
      "Epoch 50/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.3496 - acc: 0.8716 - val_loss: 0.3858 - val_acc: 0.8333\n",
      "Epoch 51/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.4412 - acc: 0.8349 - val_loss: 0.3786 - val_acc: 0.9167\n",
      "Epoch 52/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.4935 - acc: 0.8165 - val_loss: 0.5506 - val_acc: 0.7500\n",
      "Epoch 53/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.3658 - acc: 0.8807 - val_loss: 0.1609 - val_acc: 0.9167\n",
      "Epoch 54/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.3742 - acc: 0.8716 - val_loss: 0.5042 - val_acc: 0.7500\n",
      "Epoch 55/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.4075 - acc: 0.8165 - val_loss: 0.0495 - val_acc: 1.0000\n",
      "Epoch 56/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.3432 - acc: 0.8716 - val_loss: 0.1931 - val_acc: 0.9167\n",
      "Epoch 57/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.2181 - acc: 0.8991 - val_loss: 0.0488 - val_acc: 1.0000\n",
      "Epoch 58/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.3376 - acc: 0.8991 - val_loss: 0.0564 - val_acc: 1.0000\n",
      "Epoch 59/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.3737 - acc: 0.8532 - val_loss: 0.5520 - val_acc: 0.7500\n",
      "Epoch 60/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.2647 - acc: 0.8807 - val_loss: 0.0599 - val_acc: 1.0000\n",
      "Epoch 61/1000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "109/109 [==============================] - 0s 1ms/step - loss: 0.5874 - acc: 0.8257 - val_loss: 0.1719 - val_acc: 1.0000\n",
      "Epoch 62/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.2545 - acc: 0.8716 - val_loss: 0.0750 - val_acc: 1.0000\n",
      "Epoch 63/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.4132 - acc: 0.8165 - val_loss: 0.1719 - val_acc: 1.0000\n",
      "Epoch 64/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.3312 - acc: 0.8807 - val_loss: 0.1471 - val_acc: 1.0000\n",
      "Epoch 65/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.2615 - acc: 0.8991 - val_loss: 1.0510 - val_acc: 0.6667\n",
      "Epoch 66/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.4038 - acc: 0.8532 - val_loss: 0.0956 - val_acc: 1.0000\n",
      "Epoch 67/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.3192 - acc: 0.8807 - val_loss: 0.1900 - val_acc: 1.0000\n",
      "Epoch 68/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.3502 - acc: 0.8624 - val_loss: 0.1790 - val_acc: 1.0000\n",
      "Epoch 69/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.4177 - acc: 0.7982 - val_loss: 0.1185 - val_acc: 1.0000\n",
      "Epoch 70/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.2350 - acc: 0.8899 - val_loss: 0.8594 - val_acc: 0.7500\n",
      "Epoch 71/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.5457 - acc: 0.7890 - val_loss: 0.4350 - val_acc: 0.8333\n",
      "Epoch 72/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.2610 - acc: 0.8899 - val_loss: 0.0747 - val_acc: 1.0000\n",
      "Epoch 73/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.2767 - acc: 0.8899 - val_loss: 0.2572 - val_acc: 0.9167\n",
      "Epoch 74/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.2240 - acc: 0.9174 - val_loss: 0.0157 - val_acc: 1.0000\n",
      "Epoch 75/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.2348 - acc: 0.8899 - val_loss: 0.0383 - val_acc: 1.0000\n",
      "Epoch 76/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.1774 - acc: 0.8991 - val_loss: 0.0131 - val_acc: 1.0000\n",
      "Epoch 77/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.4105 - acc: 0.8440 - val_loss: 4.8734 - val_acc: 0.4167\n",
      "Epoch 78/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.7383 - acc: 0.7890 - val_loss: 0.6373 - val_acc: 0.7500\n",
      "Epoch 79/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.2928 - acc: 0.8807 - val_loss: 0.2831 - val_acc: 0.8333\n",
      "Epoch 80/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.2565 - acc: 0.8807 - val_loss: 0.0268 - val_acc: 1.0000\n",
      "Epoch 81/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.3402 - acc: 0.8532 - val_loss: 0.0240 - val_acc: 1.0000\n",
      "Epoch 82/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.1921 - acc: 0.8991 - val_loss: 0.0276 - val_acc: 1.0000\n",
      "Epoch 83/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.1528 - acc: 0.8991 - val_loss: 0.0362 - val_acc: 1.0000\n",
      "Epoch 84/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.2244 - acc: 0.8807 - val_loss: 0.0540 - val_acc: 1.0000\n",
      "Epoch 85/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.1756 - acc: 0.8991 - val_loss: 0.0072 - val_acc: 1.0000\n",
      "Epoch 86/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.1624 - acc: 0.8899 - val_loss: 0.1676 - val_acc: 0.9167\n",
      "Epoch 87/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.1907 - acc: 0.8899 - val_loss: 0.0048 - val_acc: 1.0000\n",
      "Epoch 88/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.4933 - acc: 0.7890 - val_loss: 0.3732 - val_acc: 0.8333\n",
      "Epoch 89/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.2501 - acc: 0.8991 - val_loss: 0.0939 - val_acc: 1.0000\n",
      "Epoch 90/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.1661 - acc: 0.8899 - val_loss: 0.0183 - val_acc: 1.0000\n",
      "Epoch 91/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.1689 - acc: 0.9174 - val_loss: 0.0232 - val_acc: 1.0000\n",
      "Epoch 92/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.1506 - acc: 0.9174 - val_loss: 0.0704 - val_acc: 1.0000\n",
      "Epoch 93/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.1478 - acc: 0.9083 - val_loss: 0.0090 - val_acc: 1.0000\n",
      "Epoch 94/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.5165 - acc: 0.7982 - val_loss: 4.4575 - val_acc: 0.4167\n",
      "Epoch 95/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.7020 - acc: 0.8073 - val_loss: 0.0598 - val_acc: 1.0000\n",
      "Epoch 96/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.1833 - acc: 0.9083 - val_loss: 0.0567 - val_acc: 1.0000\n",
      "Epoch 97/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.1736 - acc: 0.9358 - val_loss: 0.1912 - val_acc: 0.8333\n",
      "Epoch 98/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.1297 - acc: 0.9358 - val_loss: 0.1132 - val_acc: 1.0000\n",
      "Epoch 99/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.1677 - acc: 0.8899 - val_loss: 0.0260 - val_acc: 1.0000\n",
      "Epoch 100/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.1801 - acc: 0.9174 - val_loss: 0.0031 - val_acc: 1.0000\n",
      "Epoch 101/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.1566 - acc: 0.9083 - val_loss: 0.0032 - val_acc: 1.0000\n",
      "Epoch 102/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.1254 - acc: 0.9266 - val_loss: 0.0043 - val_acc: 1.0000\n",
      "Epoch 103/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.1459 - acc: 0.9266 - val_loss: 2.5505 - val_acc: 0.6667\n",
      "Epoch 104/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 1.5051 - acc: 0.6606 - val_loss: 1.0761 - val_acc: 0.6667\n",
      "Epoch 105/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.3638 - acc: 0.8349 - val_loss: 0.7234 - val_acc: 0.7500\n",
      "Epoch 106/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.2376 - acc: 0.8807 - val_loss: 0.5047 - val_acc: 0.7500\n",
      "Epoch 107/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.1691 - acc: 0.9174 - val_loss: 0.2127 - val_acc: 0.9167\n",
      "Epoch 108/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.1502 - acc: 0.9083 - val_loss: 0.1347 - val_acc: 0.9167\n",
      "Epoch 109/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.1199 - acc: 0.9450 - val_loss: 0.0909 - val_acc: 1.0000\n",
      "Epoch 110/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.1415 - acc: 0.9266 - val_loss: 0.3083 - val_acc: 0.7500\n",
      "Epoch 111/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.1932 - acc: 0.8991 - val_loss: 0.0073 - val_acc: 1.0000\n",
      "Epoch 112/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.0954 - acc: 0.9633 - val_loss: 0.0251 - val_acc: 1.0000\n",
      "Epoch 113/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.1611 - acc: 0.9083 - val_loss: 5.8563 - val_acc: 0.3333\n",
      "Epoch 114/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.5802 - acc: 0.7890 - val_loss: 0.0429 - val_acc: 1.0000\n",
      "Epoch 115/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.2055 - acc: 0.9174 - val_loss: 0.0309 - val_acc: 1.0000\n",
      "Epoch 116/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.1381 - acc: 0.9358 - val_loss: 0.0101 - val_acc: 1.0000\n",
      "Epoch 117/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.1683 - acc: 0.8991 - val_loss: 0.0149 - val_acc: 1.0000\n",
      "Epoch 118/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.1375 - acc: 0.9266 - val_loss: 0.0197 - val_acc: 1.0000\n",
      "Epoch 119/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.1140 - acc: 0.9541 - val_loss: 0.2428 - val_acc: 0.9167\n",
      "Epoch 120/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.2555 - acc: 0.8991 - val_loss: 0.1265 - val_acc: 0.9167\n",
      "Epoch 121/1000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "109/109 [==============================] - 0s 2ms/step - loss: 0.1031 - acc: 0.9450 - val_loss: 0.0359 - val_acc: 1.0000\n",
      "Epoch 122/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.2691 - acc: 0.8807 - val_loss: 0.0808 - val_acc: 1.0000\n",
      "Epoch 123/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.1071 - acc: 0.9633 - val_loss: 2.4382 - val_acc: 0.6667\n",
      "Epoch 124/1000\n",
      "109/109 [==============================] - 0s 2ms/step - loss: 0.1395 - acc: 0.9174 - val_loss: 0.0258 - val_acc: 1.0000\n",
      "Epoch 125/1000\n",
      "109/109 [==============================] - 0s 1ms/step - loss: 0.1043 - acc: 0.9633 - val_loss: 0.6074 - val_acc: 0.8333\n",
      "Epoch 00125: early stopping\n",
      "Fold Score (accuracy): 0.816666666667\n",
      "[[10  0  0]\n",
      " [69  1  0]\n",
      " [11 29  0]]\n",
      "[[ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 1.  0.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 1.  0.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 1.  0.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 1.  0.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  1.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  1.  0.]]\n",
      "[[  1.64455536e-03   9.98171806e-01   1.83624972e-04]\n",
      " [  4.52090857e-37   1.42218104e-08   1.00000000e+00]\n",
      " [  0.00000000e+00   4.29905452e-11   1.00000000e+00]\n",
      " [  4.51291307e-29   1.56787137e-04   9.99843240e-01]\n",
      " [  1.84045635e-16   9.99986768e-01   1.32363866e-05]\n",
      " [  0.00000000e+00   1.89148810e-14   1.00000000e+00]\n",
      " [  1.94031799e-07   9.93591309e-01   6.40846044e-03]\n",
      " [  9.46399978e-25   1.13548788e-06   9.99998808e-01]\n",
      " [  1.87010168e-21   7.03335822e-01   2.96664178e-01]\n",
      " [  6.17342943e-04   9.99034286e-01   3.48318950e-04]\n",
      " [  0.00000000e+00   6.91458826e-12   1.00000000e+00]\n",
      " [  3.10419295e-33   1.87562907e-08   1.00000000e+00]\n",
      " [  1.26517352e-25   3.41730993e-05   9.99965787e-01]\n",
      " [  1.25368964e-03   9.97980416e-01   7.65906996e-04]\n",
      " [  5.57384733e-03   9.94155645e-01   2.70491117e-04]\n",
      " [  1.19336752e-32   5.02876787e-07   9.99999523e-01]\n",
      " [  1.71560648e-28   9.99947786e-01   5.21607581e-05]\n",
      " [  2.82701515e-13   6.03256047e-01   3.96743923e-01]\n",
      " [  2.18768733e-07   9.99935269e-01   6.44933389e-05]\n",
      " [  1.19020452e-26   2.52466543e-06   9.99997497e-01]\n",
      " [  1.94145432e-05   9.98866916e-01   1.11372303e-03]\n",
      " [  7.07625935e-16   8.55140448e-01   1.44859523e-01]\n",
      " [  4.57969378e-07   9.99188483e-01   8.11047794e-04]\n",
      " [  4.51984741e-34   4.16493349e-05   9.99958396e-01]\n",
      " [  1.29209913e-21   9.95174468e-01   4.82558925e-03]\n",
      " [  4.84714005e-03   9.93808687e-01   1.34415436e-03]\n",
      " [  3.62424616e-05   9.99779403e-01   1.84406657e-04]\n",
      " [  1.14084445e-02   9.87544656e-01   1.04692369e-03]\n",
      " [  2.29524337e-02   9.75237966e-01   1.80959143e-03]\n",
      " [  0.00000000e+00   3.09338702e-07   9.99999642e-01]\n",
      " [  6.09641116e-36   1.31382303e-05   9.99986887e-01]\n",
      " [  1.50172330e-09   9.98581648e-01   1.41840859e-03]\n",
      " [  6.49803802e-02   9.34401095e-01   6.18517573e-04]\n",
      " [  9.77043328e-07   9.99626994e-01   3.72068695e-04]\n",
      " [  1.13602631e-07   9.99872684e-01   1.27212494e-04]\n",
      " [  0.00000000e+00   4.38983294e-09   1.00000000e+00]\n",
      " [  2.20212296e-05   9.99595225e-01   3.82786558e-04]\n",
      " [  3.32016424e-13   9.99554813e-01   4.45218000e-04]\n",
      " [  1.78762690e-16   7.01713502e-01   2.98286527e-01]\n",
      " [  2.00406283e-01   7.97475040e-01   2.11867364e-03]\n",
      " [  1.84248388e-01   8.13116610e-01   2.63494113e-03]\n",
      " [  1.84404414e-07   8.23157012e-01   1.76842824e-01]\n",
      " [  8.57377700e-07   9.99967337e-01   3.17694867e-05]\n",
      " [  9.61936936e-02   9.02302027e-01   1.50428258e-03]\n",
      " [  5.47224766e-11   9.99999404e-01   5.77712342e-07]\n",
      " [  5.26387291e-03   9.94376302e-01   3.59861238e-04]\n",
      " [  3.22560047e-26   2.53830931e-05   9.99974608e-01]\n",
      " [  1.63950503e-01   8.34567010e-01   1.48250198e-03]\n",
      " [  3.89128701e-07   9.99893069e-01   1.06564294e-04]\n",
      " [  1.13747437e-02   9.85572100e-01   3.05317994e-03]\n",
      " [  2.73742769e-12   9.99965668e-01   3.43739484e-05]\n",
      " [  1.04314715e-01   8.94639313e-01   1.04590040e-03]\n",
      " [  4.64277864e-01   5.33598602e-01   2.12352281e-03]\n",
      " [  0.00000000e+00   6.09923099e-15   1.00000000e+00]\n",
      " [  2.10479178e-28   3.19615356e-05   9.99968052e-01]\n",
      " [  2.59422104e-35   1.21957882e-07   9.99999881e-01]\n",
      " [  1.57328381e-04   9.99369085e-01   4.73607273e-04]\n",
      " [  2.72899792e-09   9.64805126e-01   3.51948850e-02]\n",
      " [  9.35827393e-06   9.99974966e-01   1.56433725e-05]\n",
      " [  2.56495059e-05   9.99904156e-01   7.01805620e-05]\n",
      " [  1.03682248e-06   9.99306440e-01   6.92536356e-04]\n",
      " [  9.85725546e-09   9.99996781e-01   3.21621337e-06]\n",
      " [  2.42403038e-02   9.74914968e-01   8.44758702e-04]\n",
      " [  7.59989724e-15   4.64179087e-03   9.95358169e-01]\n",
      " [  4.87120509e-01   5.10256946e-01   2.62259902e-03]\n",
      " [  3.69292124e-16   4.28177649e-04   9.99571860e-01]\n",
      " [  2.03563124e-01   7.95062184e-01   1.37468707e-03]\n",
      " [  1.36834169e-28   1.35861045e-09   1.00000000e+00]\n",
      " [  4.56473099e-05   9.98867869e-01   1.08649908e-03]\n",
      " [  1.35117653e-10   9.99482512e-01   5.17482054e-04]\n",
      " [  6.74209266e-04   9.98360097e-01   9.65712767e-04]\n",
      " [  1.09227302e-32   1.01741598e-05   9.99989867e-01]\n",
      " [  6.50242073e-05   9.99860644e-01   7.42261182e-05]\n",
      " [  1.49495023e-07   9.99992967e-01   6.93475522e-06]\n",
      " [  2.11044643e-02   9.77877557e-01   1.01804407e-03]\n",
      " [  1.57799691e-01   8.40433478e-01   1.76683476e-03]\n",
      " [  1.63826058e-07   9.99995828e-01   4.07200332e-06]\n",
      " [  6.96033986e-12   9.99953508e-01   4.64668628e-05]\n",
      " [  3.57714997e-10   9.97079492e-01   2.92053935e-03]\n",
      " [  9.97534953e-04   9.98804450e-01   1.97973684e-04]\n",
      " [  1.12450280e-08   9.99999285e-01   6.69986775e-07]\n",
      " [  6.79739820e-09   9.96438265e-01   3.56178801e-03]\n",
      " [  5.66176727e-16   8.92275155e-01   1.07724883e-01]\n",
      " [  3.41463909e-02   9.65372503e-01   4.81207389e-04]\n",
      " [  2.23165736e-21   9.85569060e-01   1.44309420e-02]\n",
      " [  1.51515730e-27   1.48865674e-03   9.98511374e-01]\n",
      " [  1.38667744e-19   9.94613707e-01   5.38631715e-03]\n",
      " [  1.57563809e-05   9.99390364e-01   5.93882403e-04]\n",
      " [  1.11532245e-07   9.99989629e-01   1.02857648e-05]\n",
      " [  2.29104310e-01   7.68877149e-01   2.01860024e-03]\n",
      " [  2.39857094e-04   9.99642611e-01   1.17458789e-04]\n",
      " [  3.69625006e-08   9.99976516e-01   2.35001189e-05]\n",
      " [  3.30657654e-11   9.98587608e-01   1.41245127e-03]\n",
      " [  1.11822687e-01   8.87216568e-01   9.60804406e-04]\n",
      " [  2.71884492e-04   9.99270022e-01   4.58131108e-04]\n",
      " [  3.58423628e-02   9.62933660e-01   1.22400839e-03]\n",
      " [  2.51788824e-36   1.31055003e-07   9.99999881e-01]\n",
      " [  3.06846661e-04   9.99600947e-01   9.22412728e-05]\n",
      " [  2.19516441e-19   5.72028339e-01   4.27971631e-01]\n",
      " [  1.77285088e-07   9.99979734e-01   2.01451912e-05]\n",
      " [  5.70867211e-02   9.40034330e-01   2.87898304e-03]\n",
      " [  5.60570684e-27   1.81025825e-04   9.99818981e-01]\n",
      " [  3.48817375e-05   9.99953032e-01   1.20256173e-05]\n",
      " [  3.00567765e-02   9.66940343e-01   3.00288922e-03]\n",
      " [  3.07790938e-26   9.82549429e-01   1.74505599e-02]\n",
      " [  1.08603192e-04   9.99852896e-01   3.84916275e-05]\n",
      " [  4.90250459e-19   1.09791001e-02   9.89020884e-01]\n",
      " [  4.77905793e-04   9.98660326e-01   8.61782348e-04]\n",
      " [  1.15963048e-03   9.97983336e-01   8.57081439e-04]\n",
      " [  6.49001668e-05   9.99515414e-01   4.19680844e-04]\n",
      " [  3.50154303e-22   9.57312584e-02   9.04268742e-01]\n",
      " [  4.74551786e-03   9.94269371e-01   9.85124265e-04]\n",
      " [  1.45509702e-07   9.90690827e-01   9.30907950e-03]\n",
      " [  1.32460371e-02   9.85645711e-01   1.10820227e-03]\n",
      " [  2.68793690e-36   9.71009655e-08   9.99999881e-01]\n",
      " [  1.34069170e-03   9.90689158e-01   7.97013380e-03]\n",
      " [  6.86071115e-04   9.98343349e-01   9.70596506e-04]\n",
      " [  2.28814055e-27   1.23678459e-04   9.99876261e-01]\n",
      " [  1.15954000e-34   2.88012689e-08   1.00000000e+00]\n",
      " [  5.69224469e-02   9.42720175e-01   3.57370271e-04]]\n",
      "[1 2 2 2 1 2 1 2 1 1 2 2 2 1 1 2 1 1 1 2 1 1 1 2 1 1 1 1 1 2 2 1 1 1 1 2 1\n",
      " 1 1 1 1 1 1 1 1 1 2 1 1 1 1 1 1 2 2 2 1 1 1 1 1 1 1 2 1 2 1 2 1 1 1 2 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 2 1 1 1 1 1 1 1 1 1 1 2 1 1 1 1 2 1 1 1 1 2 1 1 1 2\n",
      " 1 1 1 2 1 1 2 2 1]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "oos_y = []\n",
    "oos_pred = []\n",
    "fold = 0\n",
    "\n",
    "\n",
    "for train_index_full, test_index in kf.split(y_cat):\n",
    "    fold+=1\n",
    "    print(\"fold #{}\".format(fold))\n",
    "    \n",
    "    random.shuffle(train_index_full)\n",
    "    random.shuffle(test_index)\n",
    "    \n",
    "    val_number = np.rint(len(train_index_full)*0.1)\n",
    "    print(val_number, type(val_number))\n",
    "    val_index = np.random.choice(train_index_full, int(val_number), replace=False)\n",
    "    train_index = [x for x in train_index_full if x not in val_index]\n",
    "    X_train, X_val, X_test = X[train_index], X[val_index], X[test_index]\n",
    "    y_train, y_val, y_test = y_cat[train_index], y_cat[val_index], y_cat[test_index]\n",
    "    ylab_train, ylab_val, ylab_test = y[train_index], y[val_index], y[test_index]\n",
    "    z_train, z_val, z_test = z[train_index], z[val_index], z[test_index]\n",
    "    \n",
    "    \n",
    "    #Desired Augmentations\n",
    "    datagen = ImageDataGenerator(\n",
    "        fill_mode='constant',\n",
    "        horizontal_flip=True,\n",
    "        vertical_flip=True,\n",
    "        )\n",
    "    \n",
    "    #Dataset to receive augmentations (augX_train & augX_val)\n",
    "    by_train = y_train[:,0] == 1\n",
    "    by_val = y_val[:,0] == 1\n",
    "    by_test = y_test[:,0] == 1\n",
    "    augX_train, augX_val, augX_test = X_train[by_train], X_val[by_val], X_test[by_test]\n",
    "    augy_train, augy_val, augy_test = y_train[by_train], y_val[by_val], y_test[by_test]\n",
    "    \n",
    "    #Apply augmentations to augX_train & augX_val:\n",
    "    augtrainbatch_size = int((len(augX_train))*4)\n",
    "    print(augtrainbatch_size)\n",
    "    \n",
    "    \n",
    "    #for x in [0,1,2,3,4,5,6,7,8,9]:\n",
    "        #datagen.flow(x, batch_size=1)\n",
    "        #datagen.flow(augX_train[:x], batch_size=10)\n",
    "        #break\n",
    "        \n",
    "        #augimX_train = datagen.flow(x, batch_size=1)\n",
    "    \n",
    "    augimX_train = datagen.flow(augX_train, augy_train, batch_size=augtrainbatch_size)\n",
    "    #augimX_train = [for _ in datagen.flow(augX_train, augy_train, batch_size=augtrainbatch_size)]\n",
    "    print(\"augX_train:\", len(augX_train))\n",
    "    print(len(augX_train))\n",
    "    print(\"auginX_train:\", len(augimX_train))\n",
    "    print(len(augimX_train))\n",
    "    print(augimX_train)\n",
    "    #plt.imshow(augimX_train[:,0])\n",
    "    \n",
    "    #for _ in datagen.flow(augX_val):\n",
    "   \n",
    "    #Concatonate (augfullX_train -> X_train) & (augfullX_val -> X_val):\n",
    "    \n",
    "    \n",
    "    #Model:\n",
    "    model = Sequential()\n",
    "    model.add(Lambda(lambda x: x * 1./255., input_shape=(120, 160, 3), output_shape=(120, 160, 3)))\n",
    "    model.add(Conv2D(32, (3, 3), input_shape=(120, 160, 3)))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "    model.add(Conv2D(32, (3, 3)))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "    model.add(Conv2D(64, (3, 3)))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "    model.add(Flatten())  # this converts our 3D feature maps to 1D feature vectors\n",
    "    model.add(Dense(64))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dropout(0.7))\n",
    "    model.add(Dense(3))\n",
    "    model.add(Activation('softmax'))\n",
    "\n",
    "    model.compile(loss='categorical_crossentropy',\n",
    "                optimizer='rmsprop',\n",
    "                metrics=['accuracy'])\n",
    "    \n",
    "    monitor = EarlyStopping(monitor='val_loss', min_delta=1e-3, patience=25, verbose=1, mode='auto')\n",
    "   \n",
    "    \n",
    "    model.fit(\n",
    "            X_train,\n",
    "            y_train,\n",
    "            validation_data=(X_val,y_val),\n",
    "            callbacks=[monitor],\n",
    "            shuffle=True,\n",
    "            batch_size=batch_size,\n",
    "            verbose=1,\n",
    "            epochs=1000)\n",
    "        \n",
    "    pred = model.predict(X_test)\n",
    "        \n",
    "    oos_y.append(y_test)\n",
    "    #pred = np.argmax(pred)\n",
    "    predx = np.argmax(pred,axis=1)\n",
    "    oos_pred.append(predx)\n",
    "    \n",
    "    #measure the fold's accuracy:\n",
    "    y_compare = np.argmax(y_test,axis=1) #for accuracy calculation\n",
    "    score = metrics.accuracy_score(y_compare, predx)\n",
    "    print(\"Fold Score (accuracy): {}\".format(score))\n",
    "    \n",
    "    #Create confusion matrix:\n",
    "    print(confusion_matrix(ylab_test, predx))\n",
    "    \n",
    "    #Check my work:\n",
    "    print(y_test)\n",
    "    #print(test)\n",
    "    print(pred)\n",
    "    print(predx)\n",
    "    #print(z[test])\n",
    "    #print(oos_y)\n",
    "    \n",
    "    \n",
    "    #outlist = []\n",
    "    \n",
    "    #outline = index[0] + '\\t' + str(z[test]) + '\\t' + str(int(pred[0])) + '\\t' + str(int(pred[1])) + '\\t' + str(int(pred[2])) + '\\t' + str(int(y[test])) + '\\t' + str(predx) + \\n'\n",
    "    #outlist.append(outline)\n",
    "    #with open(\"results.txt\", \"w\") as f:\n",
    "        #f.writelines(outlist)\n",
    "        #f.writelines(z[test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<keras.preprocessing.image.NumpyArrayIterator at 0x7f08dfcfea50>"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#augX_train.shape\n",
    "#augX_train[:,9].shape\n",
    "#plt.imshow(augX_train[:,9])\n",
    "#datagen.flow(augX_train[:0], batch_size=1)\n",
    "datagen.flow(augX_train[:0], batch_size=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(augimX_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(y_train))\n",
    "print(y_train)\n",
    "by_train = y_train[:,0] == 1\n",
    "by_val = y_val[:,0] == 1\n",
    "by_test = y_test[:,0] == 1\n",
    "print(len(by_train))\n",
    "print(Counter(by_train).values())\n",
    "print(by_train)\n",
    "augX_train, augX_val, augX_test = X_train[by_train], X_val[by_val], X_test[by_test]\n",
    "print(len(augX_train))\n",
    "print(augX_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = cv2.imread('/home/diam/Desktop/HER2_data_aug_0/FDA_0_aug/aug_8_7339873.tif')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plt.imshow(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fhsv = cv2.cvtColor(f,cv2.COLOR_BGR2HSV)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fhsv > im.tif"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv2.imwrite('imtest.tif', fhsv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from libtiff import TIFF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15rc1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
